package cn.sccdlg.crawler.job.task;

import org.springframework.scheduling.annotation.Scheduled;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

/**
 * @author RaoShuang
 * @emil 991207823@qq.com
 * @date 2020/12/3 21:21
 *
 */
//@Component
public class ProxyTest implements PageProcessor {

    @Scheduled(fixedDelay = 10000)
    public void testProxy() {
        // 创建下载器Downloader
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        // 改下载器设置代理服务器信息
        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("60.170.111.51", 38888)));
        Spider.create(new ProxyTest())
                .addUrl("https://67ip.cn/")
                .setDownloader(httpClientDownloader)
                .run();
    }
    @Override
    public void process(Page page) {
        //打印获取到的结果以测试代理服务器是否生效
        System.out.println(page.getHtml().css("div.layui-card-body table tbody td").all());
    }
    private Site site = new Site();
    @Override
    public Site getSite() {
        return site;
    }
}
